# 搜索关键词、博文
import requests
import re
import json
import time
import random
from datetime import datetime
import datetime as dtime
import csv
from lxml import etree
from pathlib import Path

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400'
}

burl = 'https://s.weibo.com/weibo?q={}&timescope=custom:{}:{}&Refer=g&page={}'


def save_csv(csvname, row):
    print(row)
    save_path = output / f'{csvname}.csv'
    with open(save_path, 'a', encoding='utf-8', newline='') as f:
        f_csv = csv.writer(f)
        f_csv.writerow(row)


def get_page(word, day, page):
    url = burl.format(word, day, day, page)
    print(url)
    r = requests.get(url, headers=headers)
    html = etree.HTML(r.text)
    divls = html.xpath('//div[@action-type="feed_list_item"]')
    pnum = len(divls)

    if pnum == 0 or '以下是您可能感兴趣的微博' in r.text:
        return None

    print(day, page, pnum)

    for div in divls:

        mid = "".join(div.xpath("./@mid"))
        user = div.xpath('.//div[@class="info"]/div[2]/a/@href')

        if len(user) > 0:
            userlink = user[0]
            userid = re.findall('com\/(\d+)\?', userlink)[0]
            username = div.xpath(
                './/div[@class="info"]/div[2]/a/@nick-name')[0]
        else:
            username = ""
            userid = ""
        content_ = "".join(div.xpath(".//p[@node-type='feed_list_content_full']//text()")).strip()
        if content_ == "":
            content_ = "".join(div.xpath(".//p[@node-type='feed_list_content']//text()")).strip()

        time_ = div.xpath('.//div[@class="content"]/div[@class="from"]/a/text()')
        if len(time_) > 0:
            time_ = time_[0].strip()
        else:
            time_ = ""

        attid = "".join(div.xpath(".//div[@class='card-act']/ul/li[3]//text()")).strip().replace("赞", "")
        coms = "".join(div.xpath(".//div[@class='card-act']/ul/li[2]//text()")).strip().replace("评论", "")
        ret = "".join(div.xpath(".//div[@class='card-act']/ul/li[1]//text()")).strip().replace("转发", "")
        row = [username, time_, content_, userid, mid, ret, coms, attid]
        save_csv('博文_{}'.format(word), row)

    return True


def get_day(word, day):
    global hnext
    print(word, day)
    p = 0
    while True:
        try:
            hnext = get_page(word, day, p + 1)
            time.sleep(random.uniform(0, 0.05))
            p = p + 1
            if p == 50:
                break
        except:
            print('---------------------------')
            print(word, tempday, ' now:', datetime.now())
            print('---------------------------')
            time.sleep(60 * 3)
        if hnext is None:
            break




if __name__ == '__main__':




    words = ['想吃']
    headers['Cookie'] = 'SINAGLOBAL=319438869371.1958.1697274553153; SUB=_2A25IL-flDeRhGeBG7VcS9y3Nwj-IHXVr04mtrDV8PUJbkNANLWn-kW1NRhAS3AQFPsBpuQK7CiDyMa09FiqYxRbp; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhucEK.vC._Hq3xfR4Nu4mS5NHD95Qc1hqfe0M0eK.0Ws4DqcjiCrH0Uh-71Kqp; _s_tentry=weibo.com; Apache=6239890865130.7705.1697624743400; ULV=1697624743463:4:4:3:6239890865130.7705.1697624743400:1697356090320'
    output = 'weibo_files'
    output = Path(output).absolute()
    output.mkdir(exist_ok=True)


    for word in words:
        save_path = output / f'博文_{word}.csv'
        with open(save_path, 'w', encoding='utf-8-sig', newline='') as f:
            f_csv = csv.writer(f)
            f_csv.writerow(['username', 'time_', 'content_', 'userid', 'mid', 'ret', 'coms', 'attid'])
        days = ['2020-01-01', '2020-01-04']

        tempday = days[0]
        endday = datetime.strptime(days[-1], '%Y-%m-%d')

        while 1:
            print('---------------------------')
            print('now:', datetime.now())
            get_day(word, tempday)

            t = datetime.strptime(tempday, '%Y-%m-%d')
            next_day = t + dtime.timedelta(days=1)
            if next_day > endday:
                break
            tempday = next_day.strftime('%Y-%m-%d')
        print('now:', datetime.now())
